Webscraperei

Idea

Load Packages

pacman::p_load(tidyverse, wdman, jsonlite, rvest, xml2, RSelenium, here, crayon, here, magrittr)

library(RSelenium)

Paket 㤼㸱RSelenium㤼㸲 wurde unter R Version 3.4.3 erstelltWarning message:
In checkMatrixPackageVersion() : Package version inconsistency detected.
TMB was built with Matrix version 1.2.12
Current Matrix version is 1.2.11
Please re-install 'TMB' from source or ask CRAN for a binary version of 'TMB' matching CRAN's 'Matrix' package

rD

$client
  browserName
1      chrome
                                id
1 b44485ef708363f81aefc2e486dde766

$server
Process Handle
command   : C:\Users\Fabio\AppData\Local\Temp\Rtmpq0hjCh\file381c6113789b.bat 
system id : 13812
state     : running

url <- "https://www.idea.int/advanced-search?th=Direct%20Democracy%20Database"
checkboxes1 <- c("t46", "t47", "t48", "t49", "t50")
                                 "//input[@value='",jj ,"'][@type='checkbox']"

Selenium message:unknown error: Element <input type="checkbox" name="questions" data-path="46" value="t46"> is not clickable at point (53, 672). Other element would receive the click: <div id="sliding-popup" class="sliding-popup-bottom" style="height: auto; width: 100%; bottom: 0px;">...</div>
  (Session info: chrome=65.0.3325.146)
  (Driver info: chromedriver=2.36.540470 (e522d04694c7ebea4ba8821272dbef4f9b818c91),platform=Windows NT 10.0.16299 x86_64)

Error:   Summary: UnknownError
     Detail: An unknown server-side error occurred while processing the command.
     Further Details: run errorDetails method

Webscraping

library(rvest)

idea_html <- read_html("data/idea.html")

# labels <- idea_html %>% 
#   rvest::html_nodes(".country , .answer-value") %>% 
#   rvest::html_text()
# 
# codes <- idea_html %>% 
#   rvest::html_nodes(".country , .answer-value") %>% 
#   rvest::html_attr("data-nid")
# 
# idea_html %>%
#   rvest::html_nodes(".fixed-header div") %>%
#   rvest::html_text()
# 
# data.frame(labels, codes)
# 9108 / 198

cntry <- idea_html %>% 
  rvest::html_nodes(".country") %>% 
  rvest::html_text() %>% 
  rep(each=46)

codes <- idea_html %>% 
  rvest::html_nodes(".answer-value") %>% 
  rvest::html_attr("data-nid") %>% 
  as.character()

labels <- readxl::read_xls("data/idea.xls") %>% 
  names() %>% 
  .[-1] %>% 
  rep(198)

val_labs <- idea_html %>% 
  rvest::html_nodes(".answer-value") %>% 
  rvest::html_text() 

scrape_dat <- data.frame(labels, val_labs, cntry, codes, stringsAsFactors = F) %>% 
  mutate(val_labs = ifelse(val_labs == "", NA, val_labs)) %>% 
  mutate(urls = paste0("https://www.idea.int/node/", codes))

Web Crawling

Extract Dates

years <- 1776:2018 %>% 
  as.character() %>% 
  paste0(collapse = "|")

# texts <- read_html(scrape_dat$urls[1]) %>% 
#   html_nodes("#block-system-main") %>% 
#   html_text()# 

# test <- scrape_dat %>% 
#   filter(codes %in% c("200518", "226549", "229780", "0")) %>% 
#   .[1:4,] %>% 
#   mutate(urls = ifelse(codes == "0", NA, urls))

scrape_dat %<>% 
  mutate(urls = ifelse(codes == "0", NA, urls))

source("helper_function.R")



length(lastyear)

start <- 1
end <- 9108


year_dat <- scraper_of_the_year(scrape_dat, start, end) 

scraperei <- cbind(scrape_dat, year_dat)

save(scraperei, file = "data/scraperei.Rdata")
# 
# str_extract_all("hello this is one date 1999, this wouldn't be a date 3456 but this is one 1890", years)

Take a look

load("data/scraperei.Rdata")

scraperei %>% 
  filter(nyears == 2 & firstyear == lastyear) %>% 
  arrange(desc(nyears))

scraperei %>% 
  filter(nyears == 1) %>% 
  arrange(desc(nyears))

scraperei %>% 
  drop_na(firstyear)

Sudd Data

Load Packages

pacman::p_load(tidyverse, wdman, jsonlite, rvest, xml2, RSelenium, here, crayon, here, magrittr, sjmisc)

Webscraping

library(rvest)
ref_name <- read_html("data/sudd.html") %>% 
urls <- read_html("data/sudd.html") %>% 
  html_nodes(".leer a") %>% 
  html_attr("href")
dates <- read_html("data/sudd.html") %>% 
  html_nodes(".leer:nth-child(3)") %>% 
  html_text()
scrape_sudd <- data.frame(cntry, dates, ref_name, urls, stringsAsFactors = F)
read_html("data/sudd.html") %>% 
  html_nodes("td") %>% 
  html_table()

Error: html_name(x) == "table" is not TRUE

Webcrawling

start <- 1
end <- 2813
source("helper_function.R")
sudd_dat <- scraper_of_the_dats(scrape_sudd, start, end) %>%
  bind_rows()


 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=nl012018 of Country: Niederlande

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ch012018 of Country: Schweiz

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ch022018 of Country: Schweiz

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ec012018 of Country: Ecuador

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ec022018 of Country: Ecuador

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ec032018 of Country: Ecuador

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ec042018 of Country: Ecuador

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ec052018 of Country: Ecuador

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ec062018 of Country: Ecuador

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ec072018 of Country: Ecuador

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=au012017 of Country: Australien

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=it012017 of Country: Lombardei (Italien)

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=br012017 of Country: Südbrasilien (Paraná, Santa Catarina, Rio Grande do Sul)

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=es012017 of Country: Katalonien (Spanien)

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=iq012017 of Country: Irakisch-Kurdistan (Irak)

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=si012017 of Country: Slowenien

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ch062017 of Country: Schweiz

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ch072017 of Country: Schweiz

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=cl012017 of Country: Osterinsel (Rapa Nui, Isla de Pascua), Chile

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=cl032017 of Country: Osterinsel (Rapa Nui, Isla de Pascua), Chile

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=mr012017 of Country: Mauretanien

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ve012017 of Country: Venezuela

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ve022017 of Country: Venezuela

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ve032017 of Country: Venezuela

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=pr012017 of Country: Puerto Rico

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ch042017 of Country: Schweiz

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ge012017 of Country: Südossetien (Georgien)

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=hk012017 of Country: Hongkong

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=fm012017 of Country: Mikronesischer Staatenbund

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az012017 of Country: Berg-Karabach (Aserbaidschan)

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ec012017 of Country: Ecuador

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ch012017 of Country: Schweiz

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ch032017 of Country: Schweiz

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=kg012016 of Country: Kirgisistan

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=it022016 of Country: Italien

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ch132016 of Country: Schweiz

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=gd012016 of Country: Grenada

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=gd022016 of Country: Grenada

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=gd042016 of Country: Grenada

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=gd052016 of Country: Grenada

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=gd062016 of Country: Grenada

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=gd072016 of Country: Grenada

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=bg012016 of Country: Bulgarien

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=bg032016 of Country: Bulgarien

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ci012016 of Country: Elfenbeinküste

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=hu012016 of Country: Ungarn

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=co012016 of Country: Kolumbien

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=br012016 of Country: Südbrasilien (Paraná, Santa Catarina, Rio Grande do Sul)

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az022016 of Country: Aserbaidschan

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az032016 of Country: Aserbaidschan

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az042016 of Country: Aserbaidschan

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az052016 of Country: Aserbaidschan

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az062016 of Country: Aserbaidschan

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az072016 of Country: Aserbaidschan

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az092016 of Country: Aserbaidschan

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az102016 of Country: Aserbaidschan

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az122016 of Country: Aserbaidschan

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az132016 of Country: Aserbaidschan

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az142016 of Country: Aserbaidschan

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az152016 of Country: Aserbaidschan

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az162016 of Country: Aserbaidschan

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az172016 of Country: Aserbaidschan

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az182016 of Country: Aserbaidschan

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az192016 of Country: Aserbaidschan

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az202016 of Country: Aserbaidschan

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az212016 of Country: Aserbaidschan

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az222016 of Country: Aserbaidschan

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az232016 of Country: Aserbaidschan

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az242016 of Country: Aserbaidschan

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az252016 of Country: Aserbaidschan

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az262016 of Country: Aserbaidschan

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az272016 of Country: Aserbaidschan

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az282016 of Country: Aserbaidschan

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az292016 of Country: Aserbaidschan

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ch102016 of Country: Schweiz

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ch112016 of Country: Schweiz

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ba012016 of Country: Republika Srpska (Bosnien-Herzegowina)

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=li012016 of Country: Liechtenstein

 Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=zm012016 of Country: Sambia


Take a look

Test it